{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "import numpy as np\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.linear_model import LinearRegression,Ridge,LogisticRegression\n",
    "from vecstack import stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "boston = load_boston()\n",
    "X, y = boston.data, boston.target\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 1. Implementation A from scratch"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "D:\\Users\\ASUS\\anaconda3\\envs\\deeplab\\lib\\site-packages\\sklearn\\model_selection\\_split.py:297: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n  FutureWarning\n"
     ],
     "output_type": "stream"
    },
    {
     "name": "stdout",
     "text": [
      "fold 0: [3.37231835]\nfold 1: [3.20814116]\nfold 2: [3.49224784]\n\nMEAN:   [3.35756912] + [0.11645402]\nFULL:   [3.35723575]\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "# 1st level model\n",
    "model = LinearRegression()\n",
    "# Number of folds\n",
    "n_folds = 3\n",
    "# Empty array to store out-of-fold predictions (single column)\n",
    "S_train_A_scratch = np.zeros((X_train.shape[0], 1))\n",
    "# Empty array to store temporary test set predictions made in each fold\n",
    "S_test_temp = np.zeros((X_test.shape[0], n_folds))\n",
    "# Empty list to store scores from each fold\n",
    "scores = []\n",
    "# Split initialization\n",
    "kf = KFold(n_splits=n_folds, shuffle=False, random_state=0)\n",
    "\n",
    "# Loop across folds\n",
    "for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)):\n",
    "    \n",
    "    # Split data and target\n",
    "    X_tr = X_train[tr_index]\n",
    "    y_tr = y_train[tr_index]\n",
    "    X_te = X_train[te_index]\n",
    "    y_te = y_train[te_index]\n",
    "    \n",
    "    # Fit\n",
    "    _ = model.fit(X_tr, y_tr)\n",
    "    \n",
    "    # Predict out-of-fold part of train set\n",
    "    S_train_A_scratch[te_index, :] = model.predict(X_te).reshape(-1, 1)\n",
    "    \n",
    "    # Predict test set\n",
    "    S_test_temp[:, fold_counter] = model.predict(X_test)\n",
    "    \n",
    "    # Print score of current fold\n",
    "    score = mean_absolute_error(y_te, S_train_A_scratch[te_index, :])\n",
    "    scores.append(score)\n",
    "    print('fold %d: [%.8f]' % (fold_counter, score))\n",
    "    \n",
    "# Compute mean of temporary test set predictions to get final test set prediction\n",
    "S_test_A_scratch = np.mean(S_test_temp, axis=1).reshape(-1, 1)\n",
    "\n",
    "# Mean OOF score + std\n",
    "print('\\nMEAN:   [%.8f] + [%.8f]' % (np.mean(scores), np.std(scores)))\n",
    "\n",
    "# Full OOF score\n",
    "# !!! FULL score slightly differs from MEAN score because folds contain\n",
    "# different number of examples (404 can't be divided by 3)\n",
    "# If we set n_folds=4 scores will be identical for given metric\n",
    "print('FULL:   [%.8f]' % (mean_absolute_error(y_train, S_train_A_scratch)))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 3. Implementation A using vecstack"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "task:         [regression]\nmetric:       [mean_absolute_error]\nmode:         [oof_pred_bag]\nn_models:     [2]\n\nmodel  0:     [LinearRegression]\n    fold  0:  [3.37231835]\n    fold  1:  [3.20814116]\n    fold  2:  [3.49224784]\n    ----\n    MEAN:     [3.35756912] + [0.11645402]\n    FULL:     [3.35723575]\n\nmodel  1:     [Ridge]\n    fold  0:  [3.33611025]\n    fold  1:  [3.23767040]\n    fold  2:  [3.46328545]\n    ----\n    MEAN:     [3.34568870] + [0.09235564]\n    FULL:     [3.34539762]\n\n"
     ],
     "output_type": "stream"
    },
    {
     "name": "stderr",
     "text": [
      "D:\\Users\\ASUS\\anaconda3\\envs\\deeplab\\lib\\site-packages\\sklearn\\model_selection\\_split.py:297: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n  FutureWarning\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "models = [LinearRegression(),Ridge()]\n",
    "S_train_A_vecstack, S_test_A_vecstack = stacking(models, \n",
    "                                                 X_train, y_train, X_test, \n",
    "                                                 regression=True, \n",
    "                                                 n_folds=n_folds,\n",
    "                                                 mode='oof_pred_bag', \n",
    "                                                 random_state=0, \n",
    "                                                 verbose=2)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Implementation B using vecstack"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "task:         [regression]\nmetric:       [mean_absolute_error]\nmode:         [oof_pred]\nn_models:     [2]\n\nmodel  0:     [LinearRegression]\n    fold  0:  [3.37231835]\n    fold  1:  [3.20814116]\n    fold  2:  [3.49224784]\n    ----\n    MEAN:     [3.35756912] + [0.11645402]\n    FULL:     [3.35723575]\n\n    Fitting on full train set...\n\nmodel  1:     [Ridge]\n    fold  0:  [3.33611025]\n    fold  1:  [3.23767040]\n    fold  2:  [3.46328545]\n    ----\n    MEAN:     [3.34568870] + [0.09235564]\n    FULL:     [3.34539762]\n\n    Fitting on full train set...\n\n"
     ],
     "output_type": "stream"
    },
    {
     "name": "stderr",
     "text": [
      "D:\\Users\\ASUS\\anaconda3\\envs\\deeplab\\lib\\site-packages\\sklearn\\model_selection\\_split.py:297: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n  FutureWarning\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "models = [LinearRegression(),Ridge()]\n",
    "S_train_B_vecstack, S_test_B_vecstack = stacking(models, \n",
    "                                                 X_train, y_train, X_test, \n",
    "                                                 regression=True, \n",
    "                                                 n_folds=n_folds,\n",
    "                                                 mode='oof_pred', \n",
    "                                                 random_state=0, \n",
    "                                                 verbose=2)\n",
    "                                              "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}